import datetime
import wsgiref.handlers
import logging
log = logging.getLogger("dbmaint")

from google.appengine.ext import webapp
from google.appengine.ext import db
from google.appengine.api import memcache

import appdb
import spider

class UpdateTagCmd(appdb.BatchCommand):
  def entity(self, e):
    for tagName in unicode(e.tags).split("+"):
      if self.tobeAdd.has_key(appdb.GenTagKeyName(tagName)):
        tag = self.tobeAdd[appdb.GenTagKeyName(tagName)]
      else:
        tag = appdb.GetTag(tagName, True)
        if tag is None:
          tag = appdb.Tag(key_name = appdb.GenTagKeyName(tagName), name = tagName)
      tag.rank = appdb.GetRank(tag)
      if self.getEntityKeyName(e) not in tag.records:
        tag.records.append(self.getEntityKeyName(e))
        tag.rank += 1
        related = set(unicode(tag.related).split("+") + unicode(e.tags).split("+"))
        related.remove(tagName)
        related.discard("None")
        tag.related = "+".join(related)
      self.addTagIndex(tagName)
      self.tobeAdd[appdb.GenTagKeyName(tagName)] = tag

  def getEntityKeyName(self, e):
    return appdb.GenRecordKeyName(e.date, unicode(e.tags).split("+"))
  
  def getEntityName(self):
    return "ScanRecord" 
    
  def addTagIndex(self, tag):
    for word in tag:
      if self.tobeAdd.has_key(appdb.GenTagIndexKeyName(word)):
        index = self.tobeAdd[appdb.GenTagIndexKeyName(word)]
      else:
        index = appdb.GetTagIndex(word, True)
        if index is None:
          index = appdb.TagIndex(key_name = appdb.GenTagIndexKeyName(word),
                                 word = word)
      if tag not in unicode(index.tags).split("+"):
        if index.tags:
          index.tags += "+%s" %(tag)
        else:
          index.tags = tag
      self.tobeAdd[appdb.GenTagIndexKeyName(word)] = index
  
class RefreshTableCmd(appdb.BatchCommand):
  def __init__(self, pagesize):
    appdb.BatchCommand.__init__(self, pagesize)
    self.tableCmd = spider.RefreshTableSpider(1)

  def entity(self, e):
    self.tableCmd.tables.append(appdb.Table.get_by_key_name(e.value))

  def getEntityName(self):
    return "RefreshTable" 
  
  def getCondition(self):
    return ""
  
  def scanned(self, e):
    return
 
  def done(self, next, success):
    self.tableCmd.run()

class DeleteCmd(appdb.BatchCommand):
  def __init__(self, pagesize, entityName):
    appdb.BatchCommand.__init__(self, pagesize)
    self.entityName = entityName
    
  def entity(self, e):
    self.tobeDel[e.key()] = e
  
  def getEntityName(self):
    return self.entityName
  
  def getBookmark(self):
    return None
  
  def setBookmark(self, next):
    return

  def scanned(self, e):
    return

class CountCmd(appdb.BatchCommand):
  def __init__(self, pagesize, entityName, prefix):
    appdb.BatchCommand.__init__(self, pagesize)
    self.entityName = entityName
    self.prefix = prefix

  def allEntities(self, entities):
    return
  
  def getEntityName(self):
    return self.entityName
  
  def getPrefix(self):
    return self.prefix

  def scanned(self, e):
    return
  
class SortRelatedCmd(appdb.BatchCommand):
  def entity(self, e):
    related = appdb.Tag.get_by_key_name([appdb.GenTagKeyName(tag) for tag in e.related.split("+")])
    if None in related:
      raise apperr.NullTag, "Not all related tags are scanned for Tags[%s]." %(e.related.encode("utf-8"))
    related.sort(cmp = lambda x,y: cmp(y.rank, x.rank))
    e.related = "+".join([tag.name for tag in related])
    self.tobeAdd[e.key().name()] = e
    
  def getEntityKeyName(self, e):
    return appdb.GenTagKeyName(e.name)
  
  def getEntityName(self):
    return "Tag"
  
  def scanned(self, e):
    return

class CleanTagCmd(appdb.BatchCommand):
  def entity(self, e):
    if self.allNone(e.records):
      self.tobeDel[e.key().name()] = e
      appdb.DelTagFromIndex(e, self.tobeAdd, self.tobeDel)
      appdb.DelTagFromRelated(e, self.tobeAdd, self.tobeDel)
    
  def getEntityKeyName(self, e):
    return appdb.GenTagKeyName(e.name)
  
  def getEntityName(self):
    return "Tag"
  
  def scanned(self, e):
    return
  
  def allNone(self, records):
    start = 0
    batch = 1000
    while len(records[start:batch+start]) > 0:
      recs = set(appdb.Record.get_by_key_name(records[start:batch+start]))
      recs.discard(None)
      if len(recs): return False
      start += batch
    return True    

class Memcache(webapp.RequestHandler):
  def get(self):
    self.response.headers["Content-Type"] = "text/plain"
    self.response.out.write("Current Memcache:\n")
    self.response.out.write("hits:"+str(memcache.get_stats()["hits"])+"\n")
    self.response.out.write("misses:"+str(memcache.get_stats()["misses"])+"\n")
    self.response.out.write("byte_hits:"+str(memcache.get_stats()["byte_hits"])+"\n")
    self.response.out.write("items:"+str(memcache.get_stats()["items"])+"\n")
    self.response.out.write("bytes:"+str(memcache.get_stats()["bytes"])+"\n")
    self.response.out.write("oldest_item_age:"+str(memcache.get_stats()["oldest_item_age"])+"\n")
    
    if "flushall" == self.request.get("op"):
      memcache.flush_all()
      self.response.out.write("After Flush:\n")
      self.response.out.write("hits:"+str(memcache.get_stats()["hits"])+"\n")
      self.response.out.write("misses:"+str(memcache.get_stats()["misses"])+"\n")
      self.response.out.write("byte_hits:"+str(memcache.get_stats()["byte_hits"])+"\n")
      self.response.out.write("items:"+str(memcache.get_stats()["items"])+"\n")
      self.response.out.write("bytes:"+str(memcache.get_stats()["bytes"])+"\n")
      self.response.out.write("oldest_item_age:"+str(memcache.get_stats()["oldest_item_age"])+"\n")

class FlushHistory(webapp.RequestHandler):
  def get(self):
    self.response.headers["Content-Type"] = "text/plain"
    for found in [True, False]:
      key, history = appdb.GetHistory(found)
      count = len(history)
      if count:
        history = appdb.FlushHistory(history, found)
        memcache.set(key, history)
      self.response.out.write("%d %s histories flushed.\n" %(count, found))

class AddSite(webapp.RequestHandler):
  def get(self):
    url = self.request.get("url")
    if appdb.GetSite(url):
      self.response.out.write("%s already exists.\n" %(url))
      return
    site = appdb.ScanSite(key_name=appdb.GenURLKeyName("Site", url),
                          url = url)
    site.put()
    self.response.out.write("Site: %s added.\n" %(url))

class AddTable(webapp.RequestHandler):
  def get(self):
    url = self.request.get("url")
    if appdb.GetTable(url):
      self.response.out.write("%s already exists.\n" %(url))
      return
    if self.request.get("date"):
      date = datetime.datetime.strptime(self.request.get("date"), self.request.get("fmt"))
      fmt = self.request.get("fmt")
    else:
      date = datetime.datetime.now()
      fmt = "%Y"
    table = appdb.ScanTable(key_name=appdb.GenURLKeyName("Table", url),
                           url = url,
                           date = date,
                           fmt = fmt,
                           unit = self.request.get("unit"),
                           tags = self.request.get("tags"))
    table.put()
    self.response.out.write("Table: %s added.\n" %(url))

class DBBatch(webapp.RequestHandler):
  def get(self):
    cmd = None
    params = None
    pagesize = 1 if not self.request.get("pagesize") else int(self.request.get("pagesize"))
    batch = 50 if not self.request.get("batch") else int(self.request.get("batch"))
    if "site" == self.request.get("cmd"):
      cmd = spider.SiteSpider(pagesize)
    elif "table" == self.request.get("cmd"):
      cmd = spider.TableSpider(pagesize)
    elif "litetable" == self.request.get("cmd"):
      cmd = spider.LiteTableSpider(pagesize, self)
    elif "updatetag" == self.request.get("cmd"):
      cmd = UpdateTagCmd(pagesize, batch)
    elif "refreshtable" == self.request.get("cmd"):
      cmd = RefreshTableCmd(pagesize)
    elif "delete" == self.request.get("cmd"):
      cmd = DeleteCmd(pagesize, self.request.get("entity"))
    elif "sortrelated" == self.request.get("cmd"):
      cmd = SortRelatedCmd(pagesize)
    elif "cleantag" == self.request.get("cmd"):
      cmd = CleanTagCmd(pagesize)
    elif "checktable" == self.request.get("cmd"):
      cmd = spider.CheckTableSpider(pagesize, self)

    
    self.response.headers["Content-Type"] = "text/plain"
    if cmd is None:
      self.response.out.write("Command not found.\n")
      return
      
    if cmd.run():
      self.response.out.write("More command.\n")
    else:
      self.response.out.write("No more command.\n")

class DBCount(webapp.RequestHandler):
  def get(self):
    pagesize = 1 if not self.request.get("pagesize") else int(self.request.get("pagesize"))
    prefix = "Count_%s_" %(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
    cmd = CountCmd(pagesize, self.request.get("entity"), prefix)
    count = 0
    while True:
      length = cmd.run()
      count += length
      if length < pagesize: break
    self.response.out.write("%s: %d" %(self.request.get("entity"), count))
    bookmark = cmd.getBookmark()
    if bookmark: bookmark.delete()

class ViewRefreshTable(webapp.RequestHandler):
  def get(self):
    results = appdb.RefreshTable.all()
    self.response.headers["Content-Type"] = "text/plain"
    for r in results:
      self.response.out.write("%s\n" %(r.url))      

class DeleteTable(webapp.RequestHandler):
  def get(self):
    query = appdb.Record.gql("WHERE source = :1", self.request.get("url"))
    records = query.fetch(int(self.request.get("pagesize")))
    tobeAdd = {}
    tobeDel = {}
    for record in records:
      tags, date = appdb.UnGenRecordKeyName(record.key().name())
      for tag in tags:
        tag = appdb.ScratchTag(tag, tobeAdd, tobeDel)
        if tag is None \
        or record.key().name() not in tag.records: continue
        appdb.DelRecordFromTag(record.key().name(), tag, tobeAdd, tobeDel)
      tobeDel[record.key().name()] = record
    if not len(records):
      tobeDel[appdb.GenURLKeyName("Table", self.request.get("url"))] = appdb.Table.get_by_key_name(appdb.GenURLKeyName("Table", self.request.get("url")))
      tobeDel[appdb.GenURLKeyName("RT", self.request.get("url"))] = appdb.RefreshTable.get_by_key_name(appdb.GenURLKeyName("RT", self.request.get("url")))
    tobeAdd = dict(set(tobeAdd.items()) - set(tobeDel.items()))
    appdb.DBBatch(db.put, tobeAdd.values(), int(self.request.get("batch")))
    appdb.DBBatch(db.delete, tobeDel.values(), int(self.request.get("batch")))
    
    self.response.headers["Content-Type"] = "text/plain"
    if not len(records):
      self.response.out.write("No more records.\n")
    else:
      self.response.out.write("Updated[%d]:\n" %(len(tobeAdd)))
      for key in tobeAdd.keys():
        self.response.out.write("%s\n" %(key))
      self.response.out.write("\nDeleted[%d]:\n" %(len(tobeDel)))
      for key in tobeDel.keys():
        self.response.out.write("%s\n" %(key))
        
class DeleteTag(webapp.RequestHandler):
  def get(self):
    tobeAdd = {}
    tobeDel = {}
    tag = appdb.ScratchTag(self.request.get("tag"), tobeAdd, tobeDel)
    self.response.headers["Content-Type"] = "text/plain"
    if tag is None:
      self.response.out.write("Tag[%s] not found.\n" %(self.request.get("tag")))
      return
    
    tobeDel[tag.key().name()] = tag
    appdb.DelTagFromIndex(tag, tobeAdd, tobeDel)
    appdb.DelTagFromRelated(tag, tobeAdd, tobeDel)
    tobeAdd = dict(set(tobeAdd.items()) - set(tobeDel.items()))
    appdb.DBBatch(db.put, tobeAdd.values(), int(self.request.get("batch")))
    appdb.DBBatch(db.delete, tobeDel.values(), int(self.request.get("batch")))
    
    self.response.out.write("Updated[%d]:\n" %(len(tobeAdd)))
    for key in tobeAdd.keys():
      self.response.out.write("%s\n" %(key))
    self.response.out.write("\nDeleted[%d]:\n" %(len(tobeDel)))
    for key in tobeDel.keys():
      self.response.out.write("%s\n" %(key))

class RenameUnit(webapp.RequestHandler):
  def get(self):
    query = appdb.Record.gql("WHERE unit=:1", self.request.get("old"))
    records = query.fetch(int(self.request.get("pagesize")))
    self.response.headers["Content-Type"] = "text/plain"
    if not len(records):
      self.response.out.write("No more records.")
      return
    
    for record in records:
      record.unit = self.request.get("new")
    appdb.DBBatch(db.put, records)
    self.response.out.write("%d records updated." %(len(records)))
  
def main():
  application = webapp.WSGIApplication([("/dbmemcache", Memcache),
                                        ("/dbflushhistory", FlushHistory),
                                        ("/dbaddsite", AddSite),
                                        ("/dbaddtable", AddTable),
                                        ("/dbbatch", DBBatch),
                                        ("/dbcount", DBCount),
                                        ("/dbviewrefreshtable", ViewRefreshTable),
                                        ("/dbdeletetable", DeleteTable),
                                        ("/dbdeletetag", DeleteTag),
                                        ("/dbrenameunit", RenameUnit),
#                                        ("/dbrenametag", RenameTag),
                                        ],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)

if __name__ == "__main__":
  main()
